// Copyright (C) 2024 Kumo inc.
// Author: Jeff.li lijippy@163.com
// All rights reserved.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.
//

#pragma once

#include <kllm/core/types.h>
#include <llama.h>

namespace kllm {

    static bool is_valid_utf8(const std::string &str) {
        const unsigned char *bytes = reinterpret_cast<const unsigned char *>(str.data());
        const unsigned char *end = bytes + str.length();

        while (bytes < end) {
            if (*bytes <= 0x7F) {
                // 1-byte sequence (0xxxxxxx)
                bytes++;
            } else if ((*bytes & 0xE0) == 0xC0) {
                // 2-byte sequence (110xxxxx 10xxxxxx)
                if (end - bytes < 2 || (bytes[1] & 0xC0) != 0x80)
                    return false;
                bytes += 2;
            } else if ((*bytes & 0xF0) == 0xE0) {
                // 3-byte sequence (1110xxxx 10xxxxxx 10xxxxxx)
                if (end - bytes < 3 || (bytes[1] & 0xC0) != 0x80 || (bytes[2] & 0xC0) != 0x80)
                    return false;
                bytes += 3;
            } else if ((*bytes & 0xF8) == 0xF0) {
                // 4-byte sequence (11110xxx 10xxxxxx 10xxxxxx 10xxxxxx)
                if (end - bytes < 4 || (bytes[1] & 0xC0) != 0x80 ||
                    (bytes[2] & 0xC0) != 0x80 || (bytes[3] & 0xC0) != 0x80)
                    return false;
                bytes += 4;
            } else {
                // Invalid UTF-8 lead byte
                return false;
            }
        }

        return true;
    }

}  // namespace kllm
